from __future__ import division
from operator import itemgetter, attrgetter

import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np

def documentPickingAnalysisDraftOn20131108Morning():
    print "Procedure Begins..."
    xPointsList1 = []
    yPointsList1 = []
    
    xPointsList5 = []
    yPointsList5 = []
    
    xPointsList10 = []
    yPointsList10 = []
    
    xPointsList20 = []
    yPointsList20 = []
    
    xPointsList30 = []
    yPointsList30 = []

    xPointsList40 = []
    yPointsList40 = []

    xPointsList50 = []
    yPointsList50 = []

    xPointsList60 = []
    yPointsList60 = []

    xPointsList70 = []
    yPointsList70 = []

    xPointsList80 = []
    yPointsList80 = []
    
    xPointsList90 = []
    yPointsList90 = []

    xPointsList100 = []
    yPointsList100 = []    
    basePath = "/home/diaosi/web-search-engine-wei/polyIRIndexer/"
    fileNamesList = ["debugForPostingPickingAnalysis_weight_0","debugForPostingPickingAnalysis_weight_1","debugForPostingPickingAnalysis_weight_10","debugForPostingPickingAnalysis_weight_1000"]
    completeFileName = basePath + fileNamesList[3]
    inputFileHandler = open(completeFileName,"r")
    currentLine = inputFileHandler.readline()
    while currentLine:
        # "----->index1:"
        # "----->index5:"
        # "----->index10:"
        # "----->index20:"
        # "----->index30:"
        # "----->index40:"
        # "----->index50:"
        # "----->index60:"
        # "----->index70:"
        # "----->index80:"
        # "----->index90:"
        # "----->index100:"
        if currentLine.strip().startswith("----->index1:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList1.append(currentXPointValue)
            yPointsList1.append(currentYPointValue)
        if currentLine.strip().startswith("----->index5:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList5.append(currentXPointValue)
            yPointsList5.append(currentYPointValue)
        if currentLine.strip().startswith("----->index10:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList10.append(currentXPointValue)
            yPointsList10.append(currentYPointValue)
        if currentLine.strip().startswith("----->index20:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList20.append(currentXPointValue)
            yPointsList20.append(currentYPointValue)
        if currentLine.strip().startswith("----->index30:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList30.append(currentXPointValue)
            yPointsList30.append(currentYPointValue)
        if currentLine.strip().startswith("----->index40:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList40.append(currentXPointValue)
            yPointsList40.append(currentYPointValue)
        if currentLine.strip().startswith("----->index50:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList50.append(currentXPointValue)
            yPointsList50.append(currentYPointValue)
        if currentLine.strip().startswith("----->index60:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList60.append(currentXPointValue)
            yPointsList60.append(currentYPointValue)
        if currentLine.strip().startswith("----->index70:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList70.append(currentXPointValue)
            yPointsList70.append(currentYPointValue)
        if currentLine.strip().startswith("----->index80:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList80.append(currentXPointValue)
            yPointsList80.append(currentYPointValue)
        if currentLine.strip().startswith("----->index90:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList90.append(currentXPointValue)
            yPointsList90.append(currentYPointValue)
        if currentLine.strip().startswith("----->index100:"):
            # get Y-axis value
            currentYPointValue = float( inputFileHandler.readline().strip().split(" ")[1] )
            # get X-axis value
            currentXPointValue = int( inputFileHandler.readline().strip().split(" ")[1] )
            # fill the x and y list
            xPointsList100.append(currentXPointValue)
            yPointsList100.append(currentYPointValue)
        currentLine = inputFileHandler.readline()
    print "completeFileName:",completeFileName
    print "1%:"
    print "len(xPointsList1): ",len(xPointsList1)
    print "len(yPointsList1): ",len(yPointsList1)
    print
    matplotlib.pyplot.scatter(xPointsList1,yPointsList1)
    matplotlib.pyplot.show()
    
    print "5%:"
    print "len(xPointsList5): ",len(xPointsList5)
    print "len(yPointsList5): ",len(yPointsList5)
    print
    matplotlib.pyplot.scatter(xPointsList5,yPointsList5)
    matplotlib.pyplot.show()
    
    print "10%:"
    print "len(xPointsList10): ",len(xPointsList10)
    print "len(yPointsList10): ",len(yPointsList10)
    print
    matplotlib.pyplot.scatter(xPointsList10,yPointsList10)
    matplotlib.pyplot.show()
    
    print "20%:"
    print "len(xPointsList20): ",len(xPointsList20)
    print "len(yPointsList20): ",len(yPointsList20)
    print
    matplotlib.pyplot.scatter(xPointsList20,yPointsList20)
    matplotlib.pyplot.show()
    
    print "30%:"
    print "len(xPointsList30): ",len(xPointsList30)
    print "len(yPointsList30): ",len(yPointsList30)
    print
    matplotlib.pyplot.scatter(xPointsList30,yPointsList30)
    matplotlib.pyplot.show()
    
    print "40%:"
    print "len(xPointsList40): ",len(xPointsList40)
    print "len(yPointsList40): ",len(yPointsList40)
    print
    matplotlib.pyplot.scatter(xPointsList40,yPointsList40)
    matplotlib.pyplot.show()
    
    print "50%:"
    print "len(xPointsList50): ",len(xPointsList50)
    print "len(yPointsList50): ",len(yPointsList50)
    print
    matplotlib.pyplot.scatter(xPointsList50,yPointsList50)
    matplotlib.pyplot.show()
    
    print "60%:"
    print "len(xPointsList60): ",len(xPointsList60)
    print "len(yPointsList60): ",len(yPointsList60)
    print
    matplotlib.pyplot.scatter(xPointsList60,yPointsList60)
    matplotlib.pyplot.show()
    
    print "70%:"
    print "len(xPointsList70): ",len(xPointsList70)
    print "len(yPointsList70): ",len(yPointsList70)
    print
    matplotlib.pyplot.scatter(xPointsList70,yPointsList70)
    matplotlib.pyplot.show()
    
    print "80%:"
    print "len(xPointsList80): ",len(xPointsList80)
    print "len(yPointsList80): ",len(yPointsList80)
    print
    matplotlib.pyplot.scatter(xPointsList80,yPointsList80)
    matplotlib.pyplot.show()
    
    print "90%:"
    print "len(xPointsList90): ",len(xPointsList90)
    print "len(yPointsList90): ",len(yPointsList90)
    print
    matplotlib.pyplot.scatter(xPointsList90,yPointsList90)
    matplotlib.pyplot.show()    
    inputFileHandler.close()
    
    print "100%:"
    print "len(xPointsList100): ",len(xPointsList100)
    print "len(yPointsList100): ",len(yPointsList100)
    print
    matplotlib.pyplot.scatter(xPointsList100,yPointsList100)
    matplotlib.pyplot.show()
    print "Procedure Ends."

def oldDocumentPickingAnalysisWeeksAgo():
    print "Procedure Begins..."
    #poppingPostings1IncrementalPercentSet 
    #poppingPostings5IncrementalPercentSet 
    #poppingPostings10IncrementalPercentSet 
    #poppingPostings20IncrementalPercentSet 
    #poppingPostings30IncrementalPercentSet 
    #poppingPostings40IncrementalPercentSet 
    #poppingPostings50IncrementalPercentSet 
    #poppingPostings60IncrementalPercentSet 
    #poppingPostings70IncrementalPercentSet 
    #poppingPostings80IncrementalPercentSet 
    #poppingPostings90IncrementalPercentSet 
    #poppingPostings100IncrementalPercentSet 
    
    # 1% 0
    # 5% 1
    # 10% 2
    # 20% 3
    # 30% 4
    # 40% 5
    # 50% 6
    # 60% 7
    # 70% 8 
    # 80% 9 
    # 90% 10
    # 100% 11
    
    # Updated by Wei on 2013/11/01 afternoon at school
    # Q: What content are in the two lists?
    # A: the actual,total / but NOT incremental posting sets
    listOfSetsCompare1 = []
    listOfSetsCompare2 = []
    
    print "Part1:"
    poppingPostings1IncrementalPercentList = [] #index = 0
    poppingPostings5IncrementalPercentList = [] #index = 1
    poppingPostings10IncrementalPercentList = [] #index = 2
    poppingPostings20IncrementalPercentList = [] #index = 3
    poppingPostings30IncrementalPercentList = [] #index = 4
    poppingPostings40IncrementalPercentList = [] #index = 5
    poppingPostings50IncrementalPercentList = [] #index = 6
    poppingPostings60IncrementalPercentList = [] #index = 7
    poppingPostings70IncrementalPercentList = [] #index = 8
    poppingPostings80IncrementalPercentList = [] #index = 9
    poppingPostings90IncrementalPercentList = [] #index = 10
    poppingPostings100IncrementalPercentList = [] #index = 11
    listOfThePoppingList = []
    listOfThePoppingList.append(poppingPostings1IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings5IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings10IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings20IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings30IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings40IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings50IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings60IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings70IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings80IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings90IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings100IncrementalPercentList)
    
    basePath = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/"
    files = []
    files.append("allPostingsBeingPoppedAtDifferentLevels_weight_0")
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_1")
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_10")
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_20")
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_50")
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_100")
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_1000")
    for fileName in files:
        # key: the rounding value in float format
        # value: the freq of value == the key
        rawResultHistogramDict = {}
        completePath = basePath + fileName
        print "--->",completePath
        tempFileHandler = open(completePath,"r")
        listIndexCounter = 0
        tempCounter = 0
        numOfPostingsInTotal = 0
        for line in tempFileHandler.readlines():
            if line.strip().startswith("1%") or line.strip().startswith("5%") or line.strip().startswith("10%") or line.strip().startswith("20%") or line.strip().startswith("30%") or line.strip().startswith("40%") or line.strip().startswith("50%") or line.strip().startswith("60%") or line.strip().startswith("70%") or line.strip().startswith("80%") or line.strip().startswith("90%") or line.strip().startswith("100%"):
                print "listIndexCounter:",listIndexCounter
                print "len(listOfThePoppingList[listIndexCounter]):",len(listOfThePoppingList[listIndexCounter])
                listOfSetsCompare1.append( set( listOfThePoppingList[listIndexCounter] ) )
                numOfPostingsInTotal += len(listOfThePoppingList[listIndexCounter])
                print line.strip()
                
                # add lines:
                if not line.strip().startswith("100%"):
                    for element in listOfThePoppingList[listIndexCounter]:
                        listOfThePoppingList[listIndexCounter+1].append(element)
                
                listIndexCounter += 1
            elif line.strip() == "":
                pass
            else:
                listOfThePoppingList[listIndexCounter].append( line.strip() )
            tempCounter += 1
        print "tempCounter:",tempCounter
        print "numOfPostingsInTotal:",numOfPostingsInTotal
    print "len(listOfSetsCompare1):",len(listOfSetsCompare1)
    print len( listOfSetsCompare1[0].intersection( listOfSetsCompare1[0] ) )
    print len( listOfSetsCompare1[0].union( listOfSetsCompare1[0] ) )
    
    
    print "Part2:"
    poppingPostings1IncrementalPercentList = [] #index = 0
    poppingPostings5IncrementalPercentList = [] #index = 1
    poppingPostings10IncrementalPercentList = [] #index = 2
    poppingPostings20IncrementalPercentList = [] #index = 3
    poppingPostings30IncrementalPercentList = [] #index = 4
    poppingPostings40IncrementalPercentList = [] #index = 5
    poppingPostings50IncrementalPercentList = [] #index = 6
    poppingPostings60IncrementalPercentList = [] #index = 7
    poppingPostings70IncrementalPercentList = [] #index = 8
    poppingPostings80IncrementalPercentList = [] #index = 9
    poppingPostings90IncrementalPercentList = [] #index = 10
    poppingPostings100IncrementalPercentList = [] #index = 11
    listOfThePoppingList = []
    listOfThePoppingList.append(poppingPostings1IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings5IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings10IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings20IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings30IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings40IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings50IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings60IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings70IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings80IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings90IncrementalPercentList)
    listOfThePoppingList.append(poppingPostings100IncrementalPercentList)
    
    basePath = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/"
    files = []
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_0")
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_1")
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_10")
    files.append("allPostingsBeingPoppedAtDifferentLevels_weight_20")
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_50")
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_100")
    #files.append("allPostingsBeingPoppedAtDifferentLevels_weight_1000")
    for fileName in files:
        # key: the rounding value in float format
        # value: the freq of value == the key
        rawResultHistogramDict = {}
        completePath = basePath + fileName
        print "--->",completePath
        tempFileHandler = open(completePath,"r")
        listIndexCounter = 0
        tempCounter = 0
        numOfPostingsInTotal = 0
        for line in tempFileHandler.readlines():
            if line.strip().startswith("1%") or line.strip().startswith("5%") or line.strip().startswith("10%") or line.strip().startswith("20%") or line.strip().startswith("30%") or line.strip().startswith("40%") or line.strip().startswith("50%") or line.strip().startswith("60%") or line.strip().startswith("70%") or line.strip().startswith("80%") or line.strip().startswith("90%") or line.strip().startswith("100%"):
                print "listIndexCounter:",listIndexCounter
                print "len(listOfThePoppingList[listIndexCounter]):",len(listOfThePoppingList[listIndexCounter])
                listOfSetsCompare2.append( set( listOfThePoppingList[listIndexCounter] ) )
                numOfPostingsInTotal += len(listOfThePoppingList[listIndexCounter])
                print line.strip()
                
                # add lines:
                if not line.strip().startswith("100%"):
                    for element in listOfThePoppingList[listIndexCounter]:
                        listOfThePoppingList[listIndexCounter+1].append(element)
                
                listIndexCounter += 1
            elif line.strip() == "":
                pass
            else:
                listOfThePoppingList[listIndexCounter].append( line.strip() )
            tempCounter += 1
        print "tempCounter:",tempCounter
        print "numOfPostingsInTotal:",numOfPostingsInTotal
    print "len(listOfSetsCompare2):",len(listOfSetsCompare2)
    print len( listOfSetsCompare2[0].intersection( listOfSetsCompare2[0] ) )
    print len( listOfSetsCompare2[0].union( listOfSetsCompare2[0] ) )
    
    print
    print "Some Preliminary Results:"
    print "0 & 20:"
    
    print "1%:",len(listOfSetsCompare1[0]),len( listOfSetsCompare2[0].intersection( listOfSetsCompare1[0] ) ) / len( listOfSetsCompare2[0].union( listOfSetsCompare1[0] ) )
    
    print "5%:",len(listOfSetsCompare1[1]),len( listOfSetsCompare2[1].intersection( listOfSetsCompare1[1] ) ) / len( listOfSetsCompare2[1].union( listOfSetsCompare1[1] ) )
    
    print "10%:",len(listOfSetsCompare1[2]),len( listOfSetsCompare2[2].intersection( listOfSetsCompare1[2] ) ) / len( listOfSetsCompare2[2].union( listOfSetsCompare1[2] ) )
    
    print "20%:",len(listOfSetsCompare1[3]),len( listOfSetsCompare2[3].intersection( listOfSetsCompare1[3] ) ) / len( listOfSetsCompare2[3].union( listOfSetsCompare1[3] ) )
    
    print "30%:",len(listOfSetsCompare1[4]),len( listOfSetsCompare2[4].intersection( listOfSetsCompare1[4] ) ) / len( listOfSetsCompare2[4].union( listOfSetsCompare1[4] ) )
    
    print "40%:",len(listOfSetsCompare1[5]),len( listOfSetsCompare2[5].intersection( listOfSetsCompare1[5] ) ) / len( listOfSetsCompare2[5].union( listOfSetsCompare1[5] ) )
    
    print "50%:",len(listOfSetsCompare1[6]),len( listOfSetsCompare2[6].intersection( listOfSetsCompare1[6] ) ) / len( listOfSetsCompare2[6].union( listOfSetsCompare1[6] ) )
    
    print "60%:",len(listOfSetsCompare1[7]),len( listOfSetsCompare2[7].intersection( listOfSetsCompare1[7] ) ) / len( listOfSetsCompare2[7].union( listOfSetsCompare1[7] ) )
    
    print "70%:",len(listOfSetsCompare1[8]),len( listOfSetsCompare2[8].intersection( listOfSetsCompare1[8] ) ) / len( listOfSetsCompare2[8].union( listOfSetsCompare1[8] ) )
    
    print "80%:",len(listOfSetsCompare1[9]),len( listOfSetsCompare2[9].intersection( listOfSetsCompare1[9] ) ) / len( listOfSetsCompare2[9].union( listOfSetsCompare1[9] ) )
    
    print "90%:",len(listOfSetsCompare1[10]),len( listOfSetsCompare2[10].intersection( listOfSetsCompare1[10] ) ) / len( listOfSetsCompare2[10].union( listOfSetsCompare1[10] ) )
    
    print "100%:",len(listOfSetsCompare1[11]),len( listOfSetsCompare2[11].intersection( listOfSetsCompare1[11] ) ) / len( listOfSetsCompare2[11].union( listOfSetsCompare1[11] ) )
    print "Procedure Ends."

print "Program Begins..."
documentPickingAnalysisDraftOn20131108Morning()
print "Program Ends."

